# -*- coding: utf-8 -*-
import scrapy
import re
import os,sys
from scrapy.http import Request
from urllib import parse
fpath = os.path.abspath(os.path.join(os.path.dirname(__file__),".."))
ffpath = os.path.abspath(os.path.join(fpath,".."))
sys.path.append(ffpath)
from robot.items import JobBoleArticleItem
from robot.utils.common import get_md5
import datetime
import json

class JobboleSpider(scrapy.Spider):
    name = 'jobbole'
    allowed_domains = ['blog.jobbole.com']
    start_urls = ['http://blog.jobbole.com/all-posts/']
    # 数据结果
    def parse(self, response):
        '''
        1. 获取文章列表页中的文章url并交给scrapy下载并解析
        2. 获取下一页的url并交给scrapy进行下载，下载完之后交给parse解析
        #archive > div:nth-child(1) > div.post-thumb > a
        '''
        post_nodes = response.css("#archive .floated-thumb .post-thumb a")
        for post_node in post_nodes:
            # 封面图片标签获取
            image_url = post_node.css("img::attr(src)").extract_first("")
            post_url = post_node.css("::attr(href)").extract_first("")
            # 通过meta传参到response中
            yield Request(url=parse.urljoin(response.url, post_url), meta={"front_image_url": image_url},
                          callback=self.parse_detail)

        next_url = response.css(".next.page-numbers::attr(href)").extract_first("")
        if next_url:
            yield Request(url=parse.urljoin(response.url, post_url),callback=self.parse)

    def parse_detail(self,response):
        article_item = JobBoleArticleItem()
        # 文章封面图
        front_image_url = response.meta.get("front_image_url", "")
        # 文章标题
        title = response.css(".entry-header h1::text").extract()[0]
        # 文章创建的日期
        create_date = response.css("p.entry-meta-hide-on-mobile::text").extract()[0].strip().replace("·","").strip()
        # 点赞数量
        praise_nums = response.css(".vote-post-up h10::text").extract()[0]
        # 收藏数量
        fav_nums = response.css(".bookmark-btn::text").extract()[0]
        match_re = re.match(".*?(\d+).*", fav_nums)
        if match_re:
            fav_nums = int(match_re.group(1))
        else:
            fav_nums = 0
        # 评论数量
        comment_nums = response.css("a[href='#article-comment'] span::text").extract()[0]
        match_re = re.match(".*?(\d+).*", comment_nums)
        if match_re:
            comment_nums = int(match_re.group(1))
        else:
            comment_nums = 0
        # 文章内容
        content = response.css("div.entry").extract()[0]
        # 文章所属类型  IT技术.Linux
        tag_list = response.css("p.entry-meta-hide-on-mobile a::text").extract()
        tag_list = [element for element in tag_list if not element.strip().endswith("评论")]
        tags = ",".join(tag_list)

        article_item['url_object_id'] = get_md5(response.url.encode("utf-8"))
        article_item['title'] = title
        article_item['front_image_url'] = [front_image_url]

        # 通过datetime模块将字符串的date转换为date类型的数据
        try:
            create_date = datetime.datetime.strptime(create_date,"%Y/%m/%d").date()
        except Exception as e:
            create_date = datetime.datetime.now().date()
        print("---",create_date)
        article_item['create_date'] = create_date
        article_item['url'] = response.url
        article_item['praise_nums'] = praise_nums
        article_item['fav_nums'] = fav_nums
        article_item['comment_nums'] = comment_nums
        article_item['content'] = content
        article_item['tags'] = tags
        # 通过执行yield,此时会将item交给pipelines执行
        yield article_item

